#ifndef _PGLBPPRO_H_
#define _PGLBPPRO_H_

#include "WebPage.h"
#include <fstream>
#include <unordered_map>
#include <utility>

using std::ifstream;
using std::unordered_map;

class PageLibPreProcessor {
  public:
    PageLibPreProcessor(SplitTool *);
    // PageLibPreProcessor();

    void cutReddundantPage();
    void buildInvertIndexMap();
    void storeOnDisk();
    void storeIndexOnDisk();

  private:
    void wordParticiple(const string &); // 分词

  private:
    vector<WebPage> _pageList;
    unordered_map<int, pair<int, int>> _offsetLib;
    unordered_map<string, set<pair<int, double>>> _invertIndexLib;
    SplitTool *_wordCutter;

    vector<map<string, int>> _tfList;        // 每篇文章的词频
    unordered_map<string, int> _unique_word; // 每个词的词频
};

#endif